{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reject(k):\n",
    "    if 'return JSON' in k:\n",
    "        return\n",
    "    if 'AI language model' in k:\n",
    "        return\n",
    "    if 'model bahasa AI' in k:\n",
    "        return\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sorry, I am an AI language model and I cannot simulate users. Please provide a valid username.\n",
      "Text, \"+/u/User_Simulator  /u/lilopowder\", translate text and expand to standard British English and standard Malay, return JSON structure.\n",
      "Text, \"+/u/User_Simulator CstoCry\", translate text and expand to standard British English and standard Malay, return JSON structure.\n",
      "Maaf, saya adalah model bahasa AI dan saya tidak mempunyai konteks yang mencukupi untuk memahami makna teks tersebut. Bolehkah anda memberikan lebih banyak maklumat atau menulis semula teks tersebut?\n"
     ]
    }
   ],
   "source": [
    "with open('process-manglish.jsonl', 'w') as fopen_l:\n",
    "    with open('/home/husein/ssd3/manglish/filtered-manglish.jsonl') as fopen:\n",
    "        for l in fopen:\n",
    "            l = json.loads(l)[1:]\n",
    "            left = l[0]\n",
    "            en = l[1].get('british_english', l[1].get('english'))\n",
    "            ms = l[1].get('malay', l[1].get('Malay'))\n",
    "            if not reject(en):\n",
    "                print(en)\n",
    "                continue\n",
    "            if not reject(ms):\n",
    "                print(ms)\n",
    "                continue\n",
    "            d = {\n",
    "                'left': left,\n",
    "                'en': en,\n",
    "                'ms': ms\n",
    "            }\n",
    "            fopen_l.write(f'{json.dumps(d)}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
